import java.io.BufferedWriter;
import java.io.FileWriter;
import java.util.ArrayList;
import entity.WebPage;


public class CrawlerClient extends Thread{
	
	QueueServer server; 
	LuceneIndexer indexer;
	BufferedWriter out;
	
	
	public CrawlerClient() {
		
		server = QueueServer.getInstance();
		indexer = LuceneIndexer.getInstance();
		try {
		FileWriter fstream = new FileWriter("D:\\lucene\\log.txt");
		out = new BufferedWriter(fstream);	
		
		}catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public void run() {
		
		
		int numOfRecords = 0;
		
		while(numOfRecords < 2000) {
			//urllist will contain all the url that is to be parse.
			ArrayList urlList = server.getNextUrlBatch();
			WebPage webPage = null;
						
			if(urlList != null || urlList.size() != 0) {
				
				for(int i = 0; i < urlList.size(); i++) {
					String url = (String) urlList.get(i);
					if(url.length() != 0) {
						//the downloadCrawlerResult with return a web page object with its title, decsriptiom, url and additional link parse.
						webPage = new Downloader().downloadCrawlerResult(url);
						
						if(webPage != null) {
							
							//index the url1
							
							if(indexer.indexUrl(webPage)) {
									numOfRecords++;
							
								try {
									out.write("URL INDEXED : "+webPage.getUrl());
									out.newLine();
									out.write("NUM OF RECORD : " + numOfRecords);
									out.newLine();
									out.flush();
									
								}catch(Exception e) {
									e.printStackTrace();
								}
							}
							
							
							//add the additional link parse on the page to the queue server.
							ArrayList additionalLink = webPage.getUrlList();
							
							for(int o = 0; o < additionalLink.size(); o++) {
								String link = (String) additionalLink.get(o);
								server.addUrl(link);
							}
						}
					}
					
				}
			}
			else {
				//slpz for 10sec before asking again
				System.out.println("Sleeping");
			}
	}	//end while
		indexer.close();
		
		
	}//end main
	
}


